﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Data.SqlClient;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Threading.Tasks;
using System.Text.RegularExpressions;
using System.Net.Mail;

namespace Articleulator3
{
	public partial class Articleulator : Form
	{
		private const int NumParallelism = 6;
		private string connstr = System.Configuration.ConfigurationManager.ConnectionStrings["SQL"].ConnectionString;
		private const string BloggerProject = "806534914786", bloggerClient = "806534914786.apps.googleusercontent.com", BloggerClientSecret = "gAt9XE9fQIt0fIaR_m3uMJtr", bloggerRedirectURI = "urn:ietf:wg:oauth:2.0:oob", BloggerAPIKey = "AIzaSyDVW1PNEc01wumLXimJfFlq9Drjqu_04Y4",
			BighugelabsApiKey = "26265a805eba5b431dc4f133cf45a3e2"; // http://words.bighugelabs.com/admin/
		private volatile DataRow[] dtSynonyms = new DataRow[0];
		private DateTime SynoLookup = DateTime.MinValue, ThesaurusLookupReset = DateTime.MinValue;
		private int ThesaurusLookups = 0;

		public Articleulator()
		{
			InitializeComponent();
		}

		private void btnStart_Click(object sender, EventArgs e)
		{
			txtResult.Text = "";
			bgw.RunWorkerAsync(new object[] { txtSubject.Text.Trim(), txtGmailPass.Text.Trim() });
		}

		private void bgw_DoWork(object sender, DoWorkEventArgs e)
		{
			object[] args = (object[])e.Argument;

			string subject = (string)args[0], lower = subject.ToLower(), gPass = (string)args[1];
			if (subject.Length == 0) return;

			// Go find some pages matching this subject
			bgw.ReportProgress(0, "Searching for links");
			string SearchHTML = "";
			int numPagesToGet = 10;
			List<string> urls = new List<string>();
			for (int i = 0; urls.Count < numPagesToGet; i++) {
				//bgw.ReportProgress((int)(((double)i / (double)urls.Count) * 100), "Looking up URLS (" + i.ToString() + " search pages, " + urls.Count.ToString() + " of " + numPagesToGet.ToString() + " URLs found)");
				SearchHTML += GetSearchHTML(subject, i);
				urls.AddRange(GetURLsFromSearchHTML(SearchHTML, 10)); }

			// Extract all sentences
			List<string> sentences = new List<string>();
			bgw.ReportProgress(25, "Extracting sentences");
			Parallel.For(0, urls.Count, i => {
				//bgw.ReportProgress(((int)((double)i / (double)urls.Count) * 100), "Downloading " + urls[i]);
				string text = null;
				try { text = new System.Net.WebClient().DownloadString(urls[i]); } catch { return; }
				List<string> sentences2 = ExtractSentences(text);
				sentences.AddRange(sentences2); });

			 //Replace some words with synonyms
			bgw.ReportProgress(50, "Replacing synonyms");
			for (int i = 0; i < sentences.Count; i++)
			    sentences[i] = ReplaceSynonyms(sentences[i]);

			// Build paragraphs
			bgw.ReportProgress(75, "Building paragraphs");
			List<string> paragraphs = new List<string>();
			paragraphs.Add("");
			int curParaCount = 0;
			for (int i = 0; i < sentences.Count; i++) {
				//bgw.ReportProgress((int)((((double)i) / ((double)sentences.Count)) * 100), "Building paragraphs");
				if (curParaCount > 5) {
					curParaCount = 0;
					paragraphs.Add(""); }
				paragraphs[paragraphs.Count - 1] += sentences[i] + "  ";
				curParaCount++; }

			// Produce result
			bgw.ReportProgress(80, "Authoring article");
			string body = "";
			foreach (string paragraph in paragraphs)
				body += paragraph + "\r\n\r\n";
			while (body.Contains("   ")) body = body.Replace("   ", "  ");

			// Publish article
			using (MailMessage message = new MailMessage()) {
				message.From = new MailAddress("tsilb3@gmail.com", "Kevin Connolly");
				message.Subject = subject;
				message.Body = body;
				message.SubjectEncoding = Encoding.UTF8;
				message.BodyEncoding = Encoding.UTF8;
				message.IsBodyHtml = false;
				message.To.Add("tsilb3.ArticlePublishment@blogger.com");

				SmtpClient client = new SmtpClient("smtp.gmail.com");
				client.EnableSsl = true;
				client.Port = 587;
				client.UseDefaultCredentials = false;
				client.Credentials = new System.Net.NetworkCredential("tsilb3@gmail.com", gPass);
				client.Send(message); }

			bgw.ReportProgress(100, "Done");
		}
		private void bgw_ProgressChanged(object sender, ProgressChangedEventArgs e)
		{
			if (!bgw.IsBusy) {
				lblProgress.Text = "";
				pbProgress.Value = 0;
				return; }
			if (e.ProgressPercentage > 0) pbProgress.Value = e.ProgressPercentage;
			if (e.UserState != null) {
				lblProgress.Text = (string)e.UserState;
				txtResult.Text += (string)e.UserState + "\r\n"; }
		}
		private void bgw_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
		{
		}

		private string GetSearchHTML(string subject, int Page = 1)
		{
			string SearchURL = @"http://search.yahoo.com/search?p=" + System.Web.HttpUtility.UrlEncode(subject);
			if (Page != 1) SearchURL += @"&pstart=" + Page.ToString() + "&b=" + ((Page * 10) + 1).ToString();
			return new System.Net.WebClient().DownloadString(SearchURL);
		}
		private string[] GetURLsFromSearchHTML(string SearchHTML, int numResults)
		{
			List<string> results = new List<string>();
			if (!SearchHTML.Contains("")) return null;
			SearchHTML = SearchHTML.Substring(SearchHTML.IndexOf("<h2>Search results</h2>"));
			for (int i = 0; i < SearchHTML.Length; i++) {
				//bgw.ReportProgress((int)(((double)i / (double)SearchHTML.Length) * 100), "Parsing links");
				string substring = SearchHTML.Substring(i);
				string searchkey = "<a id=\"link-";
				if (!substring.StartsWith(searchkey)) continue;
				if (results.Count >= numResults) break;
				string url = substring.Substring(substring.IndexOf(searchkey) + 6);
				url = url.Substring(url.IndexOf("href=\"") + 6);
				url = url.Substring(0, url.IndexOf("\"") - 1);
				i += searchkey.Length + url.Length;
				results.Add(url);
			}
			return results.ToArray();
			// http://search.yahoo.com/r/_ylt=A0oGdXEcqm1Qf2YACg9XNyoA;_ylu=X3oDMTE0ODJ2YTduBHNlYwNzcgRwb3MDMQRjb2xvA3NrMQR2dGlkA1ZJUDA1MV83Ng--/SIG=11qce35e6/EXP=1349393052/**http%3a//www.pearsonhighered.com/mytest/
			// <h2>Search results</h2><ol start="1" data-bns="API"><li><div class="res"><div><h3><a id="link-1" class="yschttl spt" 
			//	href="/r/_ylt=A0oGdXEcqm1Qf2YACg9XNyoA;_ylu=X3oDMTE0ODJ2YTduBHNlYwNzcgRwb3MDMQRjb2xvA3NrMQR2dGlkA1ZJUDA1MV83Ng--/SIG=11qce35e6/EXP=1349393052/**http%3a//www.pearsonhighered.com/mytest/"target="_blank" data-bk="5058.1"><b>MyTest</b> 
			//| Pearson</a></h3></div><div class="abstr">Welcome. Complement

			// <h2>Search results</h2><ol start="1" data-bns="API"><li><div class="res"><div><h3>
			// <a id="link-1" class="yschttl spt" href="http://www.pearsonhighered.com/mytest/"
		}
		public static List<string> ExtractSentences(string plaintext)
		{
			if (string.IsNullOrEmpty(plaintext)) throw new ArgumentNullException("plaintext");
			if (!plaintext.ToLower().Contains("<body")) return new List<string>();
			plaintext = plaintext.Substring(plaintext.ToLower().IndexOf("<body"));
			while (!plaintext.StartsWith(">")) plaintext = plaintext.Substring(1);
			plaintext = ConvertDownloadPageRowToPlaintext(plaintext.Replace("â€™", "'").Replace("â", "").Replace("Â", "").Replace("€", "").Replace("˜", "").Trim());

			// Format sentences
			plaintext = plaintext.Replace(".", ".│").Replace("?", "?│").Replace("!", "!│").Replace(":", ":│");
			string[] sentenceCandidates = plaintext.Split(new string[] { "│" }, StringSplitOptions.RemoveEmptyEntries);
			for (int i=0; i < sentenceCandidates.Length; i++) {
				string result = sentenceCandidates[i];
				if (result.Length < 20) continue; // result is too short
				if (result.Length > 300) continue; // result is too long
				while (result.Where(c => c == '(').Count() > result.Where(c => c == ')').Count()) result += ")";
				if (result.Where(c => c == '(').Count() != result.Where(c => c == ')').Count()) continue; // result has a mismatched # of parens
				if (result.Split(' ').Length < 4) continue; // result has too few words
				if ((double)Regex.Matches(result.Replace(" ", ""), @"([a-zA-Z\,\;\(\)]{1})", RegexOptions.Singleline).Count / (double)result.Replace(" ", "").Length < .7D) continue; // result has a low % of letters; likely math
				sentenceCandidates[i] = result;
			}

			// Find beginning and end of likely content
			int first = 0, last = sentenceCandidates.Length;
			for (int i = 0; i < sentenceCandidates.Length && first == 0; i++) if (sentenceCandidates[i].Length > 40) first = i;
			for (int i = sentenceCandidates.Length - 1; i >= first && last == sentenceCandidates.Length; i--) if (sentenceCandidates[i].Length > 40) last = i;
			while (last - first < 5 && last < sentenceCandidates.Length) last++;

			// Concatenate list of sentences
			List<string> sentences = new List<string>();
			for (int i = first; i < last; i++) {
				if (sentenceCandidates[i].Length < 30 || sentenceCandidates[i].Length > 300) continue;
				if (sentenceCandidates[i].ToCharArray().Distinct().Count() < 10) continue;
				bool stop = false;
				foreach (string err in new string[] { "error", "javascript", "be blank", "accessing", "acknowledge", "copyright", "©", "|", "—", "thank", "welcome", "input", "/", "\\" })
					if (sentenceCandidates[i].ToLower().Contains(err)) { stop = true; break; }
				if (!stop) sentences.Add(sentenceCandidates[i].Trim()); }

			// Capitalize the first letter of every sentence and make the rest lowercase
			for (int i = 0; i < sentences.Count; i++)
				sentences[i] = sentences[i][0].ToString().ToUpper() + sentences[i].Substring(1).ToLower();

			return sentences;
		}
		public static List<string> ExtractSentencesFromPlaintext(string plaintext)
		{
			if (string.IsNullOrEmpty(plaintext)) throw new ArgumentNullException("plaintext");

			// Format sentences
			plaintext = plaintext.Replace(".", ".│").Replace("?", "?│").Replace("!", "!│").Replace(":", ":│");
			string[] sentences = plaintext.Split(new string[] { "│" }, StringSplitOptions.RemoveEmptyEntries);
			for (int i = 0; i < sentences.Length; i++) {
				string result = sentences[i];
				if (result.Length < 20) continue; // result is too short
				if (result.Length > 300) continue; // result is too long
				while (result.Where(c => c == '(').Count() > result.Where(c => c == ')').Count()) result += ")";
				if (result.Where(c => c == '(').Count() != result.Where(c => c == ')').Count()) continue; // result has a mismatched # of parens
				if (result.Split(' ').Length < 4) continue; // result has too few words
				if ((double)Regex.Matches(result.Replace(" ", ""), @"([a-zA-Z\,\;\(\)]{1})", RegexOptions.Singleline).Count / (double)result.Replace(" ", "").Length < .7D) continue; // result has a low % of letters; likely math
				sentences[i] = result; }

			// Capitalize the first letter of every sentence and make the rest lowercase
			for (int i = 0; i < sentences.Length; i++)
				sentences[i] = sentences[i][0].ToString().ToUpper() + sentences[i].Substring(1).ToLower();

			return sentences.ToList();
		}
		private string ReplaceSynonyms(string text)
		{
			string result = text;
			char[] seperators = new char[] { ' ', '.', ',', ';', ':', '!', '(', ')' };
			foreach (string word in text.Split(seperators))
				if (word.Trim().Length > 4) try { result = result.Replace(word, ReplaceWord(word)); } catch { }
			return result;
		}
		private string ReplaceWord(string word)
		{
			if (SynoLookup < DateTime.Now.AddMinutes(-30))
				RefreshSynonyms();
			bool matched = false;
			foreach (DataRow dr in dtSynonyms.Where(r => r["word"].ToString().ToLower() == word.ToLower() || r["Synonym"].ToString().ToLower() == word.ToLower()))
				if (dr["Word"].GetType() == typeof(DBNull) || dr["Synonym"].GetType() == typeof(DBNull)) continue; // Found to have no synonyms
				else if (dr["Word"].ToString() != word.ToLower()) return dr["Word"].ToString();
				else if (dr["Synonym"].ToString() != word.ToLower()) return dr["Synonym"].ToString();
				
			if (!matched) {// Go look up a new one
				string syn = LookupSynonymAPI(word);
				if (syn != null) return syn;  }
			return word;
		}

		private void RefreshSynonyms()
		{
			dtSynonyms = CallSql_Query("select Word, Synonym from Synonyms order by newid()").Select(); // where '" + word.Replace("'", "''") + "' in (word, synonym) 
			SynoLookup = DateTime.Now;
			bgw.ReportProgress(-1, "Refreshed synonyms; " + dtSynonyms.Length.ToString() + " on file.");
		}
		private string LookupSynonymAPI(string word)
		{	
			if (ThesaurusLookups >= 10000) return word; // API daily limit
			foreach (char c in word)
				if (new string[] { "&","?" }.Contains(c.ToString())) return word;
			string url = "http://words.bighugelabs.com/api/2/" + BighugelabsApiKey + "/" + word + "/";
			string ApiResult = null;
			bool HasResult = false;
			try { ApiResult = new System.Net.WebClient().DownloadString(url); HasResult=true;}  catch { }
			if (ThesaurusLookupReset < DateTime.Today) ThesaurusLookupReset = DateTime.Now;
			ThesaurusLookups++;

			if (HasResult) {
				string query = "";
				foreach (string resultline in ApiResult.Trim().Split('\n')) // e.g.:  noun|syn|yield
					query += "insert into Synonyms (Word, Synonym, WordType, isAntonym) values ('" + word.Replace("'", "''") + "', '" + resultline.Split('|')[2].Replace("'", "''") + "', '" + resultline.Split('|')[0].Replace("'", "''") + "', " + (resultline.Split('|')[1] == "ant" ? "1" : "0") + "); ";
				CallSql_Query(query);
				RefreshSynonyms();
				bgw.ReportProgress(-1, "Looked up " + ApiResult.Trim().Split('\n').Length.ToString() + " synonyms for " + word);
				return ApiResult.Split('\n')[0].Split('|')[2]; }
			else {
				CallSql_Query("insert into Synonyms (Word, Synonym) values ('" + word.Replace("'","''") + "', null)");
				bgw.ReportProgress(-1, "Looked up 0 synonyms for " + word); }
			return null;
		}

		public static string ConvertDownloadPageRowToPlaintext(string body)
		{
			string result = "";
			if (body.Contains("<body")) body = body.Substring(body.IndexOf(">", body.IndexOf("<body") + 1) + 1);
			if (body.Contains("</body")) body = body.Substring(0, body.IndexOf("</body"));
			body = body.Replace("<br>", "\r\n\r\n").Replace("<br />", "\r\n\r\n").Replace("<p>", "\r\n\r\n").Replace("</h1>", "</h1>\r\n\r\n").Replace("</h2>", "</h1>\r\n\r\n").Replace("</h3>", "</h1>\r\n\r\n").Replace("</h4>", "</h1>\r\n\r\n").Replace("</h5>", "</h1>\r\n\r\n");

			// Parse HTML, locate and extract plaintext
			DateTime started = DateTime.Now;
			for (int i = 0; i < body.Length; i++) // skip all HTML and tags' inner contents
				try { // Stage 1
					if (DateTime.Now - started > TimeSpan.FromSeconds(5)) break;
					string sub = body.Substring(i);
					if (sub.StartsWith("<!--")) { i = body.IndexOf("-->", i + 1) + 2; continue; }
					else if (sub.StartsWith("</")) { i = body.IndexOf(">", i + 1); continue; }
					else if (sub.StartsWith("<style") || sub.StartsWith("<a ") || sub.StartsWith("<sup") || sub.StartsWith("<sub")) { i = body.IndexOf(">", body.IndexOf(">", i + 1) + 1); continue; }
					else if (sub.StartsWith("<script")) { i = body.IndexOf("</script>", body.IndexOf(">", i + 1) + 1) + 8; continue; }
					else if (sub.StartsWith("<div") || sub.StartsWith("<p ") || sub.StartsWith("<span")) { i = body.IndexOf(">", i + 1); continue; }
					else if (sub.StartsWith("/>")) { i += 1; continue; }
					else if (sub.StartsWith("/ >")) { i += 1; continue; }
					else if (sub.StartsWith("<")) { i = body.IndexOf(">", i + 1); continue; }
					if (i >= body.Length) break;
					result += body[i]; }
				catch { i++; }

			// Stage 2:  Format result document to facilitate sentence extrapolation and word extraction
			try {
				result = Regex.Replace(result, "\\[(.*)\\]", "");
				result = result.Trim();
				result = System.Web.HttpUtility.HtmlDecode(result);
				result = System.Web.HttpUtility.UrlDecode(result);
				result = result.Replace("\t", " ").Replace("\r", "\n").Replace("\n", ".").Replace("·", " ").Replace("∙", " ").Replace("·", " ").Replace(":", " ");
				while (result.Contains("  ")) result = result.Replace("  ", " ");
				while (result.Contains("..")) result = result.Replace("..", ".");
				while (result.Contains(". ")) result = result.Replace(". ", ".");
				while (result.Contains("..")) result = result.Replace("..", ".");
				while (result.Contains("  ")) result = result.Replace("  ", " "); }
			catch { }

			// Stage 3
			result = result.Replace("<", "").Replace(">", "");
			foreach (char c in new char[] { ',', '.', ';', ':', '/' })
				while (result.Contains(c.ToString() + c.ToString()))
					result = result.Replace(c.ToString() + c.ToString(), c.ToString());
			result = result.Replace(".", ".  ").Replace(",", ", ");

			return result;
		}
		public DataTable CallSql_Query(string query)
		{
			if (string.IsNullOrEmpty(query)) throw new ArgumentNullException("query");

			DataTable dt = new DataTable();
			using (SqlConnection con = new SqlConnection(connstr))
			using (SqlDataAdapter da = new SqlDataAdapter(query, con))
				da.Fill(dt);
			return dt;
		}

		private void btnManual_Click(object sender, EventArgs e)
		{
			//Replace some words with synonyms
			string body = txtManual.Text.Trim();

			List<string> sentences = ExtractSentencesFromPlaintext(body);
			for (int i = 0; i < sentences.Count; i++)
			    sentences[i] = ReplaceSynonyms(sentences[i]);

			// Build paragraphs
			List<string> paragraphs = new List<string>();
			paragraphs.Add("");
			int curParaCount = 0;
			for (int i = 0; i < sentences.Count; i++) {
				if (curParaCount > 5) {
					curParaCount = 0;
					paragraphs.Add(""); }
				paragraphs[paragraphs.Count - 1] += sentences[i] + "  ";
				curParaCount++; }

			// Produce result
			string body2 = "";
			foreach (string paragraph in paragraphs)
				body2 += paragraph + "\r\n\r\n";
			while (body2.Contains("   ")) body2 = body2.Replace("   ", "  ");
			txtManual.Text = body2;
		}
	}
}
